#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
爬取国家统计局最新地址库
省市区三级（Json版本）
author: xiexie1993
time: 2020-07-22
"""

import requests
from bs4 import BeautifulSoup
import json
import os
import datetime


def get_province(index_href):
    """抓取省份信息"""
    # 创建字典
    json_data = {}
    province_url = url + index_href
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
    }
    request = requests.get(province_url, headers=headers)
    #  print('[Debug] request=', request)
    request.encoding = 'gbk'
    province_html_text = str(request.text)
    #  print('[Debug] province_html_text=', province_html_text)
    soup = BeautifulSoup(province_html_text, "html.parser")
    #  print('[Debug] soup=', soup)
    province_tr_list = soup.select('.provincetr a')
    #  print('[Debug] province_tr_list=', province_tr_list)
    #  province_list = {}
    province_lists = []
    province_key = 0
    city_lists = {}
    city_count = 0
    # 遍历省份列表信息
    for province_tr in province_tr_list:
        if province_tr:
            province_href = province_tr.attrs['href']
            #  print('[Debug] province_href=', province_href)
            province_no = province_href.split('.')[0]
            #  print('[Debug] province_no=', province_no)
            province_code = province_no + '0000'
            #  print('[Debug] province_code=', province_code)
            province_name = province_tr.text
            #  print('[Debug] province_name=', province_name)
            #  province_info = {'code': province_code, 'name': province_name}
            province_info = {'text': province_name, 'value': province_code}
            #  print('[Debug] province_info=', province_info)
            #  province_list.setdefault(province_key, province_info)
            province_lists.append(province_info)
            #  print('[Debug] province_lists==', province_lists)
            province_key += 1
            print("[Debug] 开始抓取第%s个省:%s,编码：%s" %(province_key,province_name,province_code))
            #  print('新增数据：', province_info)
            # 市级
            city_list = []
            city_list = get_city(province_href, province_code)
            #  print('[Debug] city_lists=',city_list)
            city_count = city_count + len(city_list)
            print('[Debug] city_count=',city_count)
            #  city_lists.append(city_list)
            city_lists[province_code]=city_list

    #  print('[INFO]抓取省份下市信息结束！')
    print("[INFO] 抓取%s个省" %(len(province_lists)))
    print("[INFO] 抓取%s个市" %(city_count))
    # 赋值写入文件数据
    #  json_data = province_lists
    json_data = city_lists
    #  print('[Debug] json_data=',json_data)
    #  nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    nowTime = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    filename = "json2019/city" + str(nowTime) + ".json"
    print('[Debug] filename=',filename)
    file = open(filename, 'w', encoding='utf-8')
    json.dump(json_data, file, skipkeys=False,ensure_ascii=False,indent=4)
    file.close()
    print('[INFO]数据写入完成！')

def get_city(province_href, province_code):
    """抓取市级城市信息"""
    print('[INFO]开始抓取编码为%s的省下的市级信息' %(province_code))
    #  print('[Debug] province_href=', province_href)
    #  print('[Debug] province_code=', province_code)
    city_url = url + province_href
    #  print('[Debug] city_url=', city_url)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
    }
    request = requests.get(city_url, headers=headers)
    #  print('[Debug] request=', request)
    request.encoding = 'gbk'
    city_html_text = str(request.text)
    #  print('[Debug] city_html_text=', city_html_text)
    soup = BeautifulSoup(city_html_text, "html.parser")
    #  print('[Debug] soup=', soup)
    city_tr_list = soup.select('.citytr')
    #  print('[Debug] city_tr_list=', city_tr_list)
    #  if os.path.exists('json2019/city.json'):
    #      file = open('json2019/city.json', 'r', encoding='utf-8')
    #      city_data = json.load(file)
    #      #  print('[Debug] city_data=', city_data)
    #      file.close()
    #  else:
    #      city_data = {}
    #      #  city_data = []
    #  print('[Debug] city_data=', city_data)
    city_list = []
    #  print('[Debug] city_list=', city_list)
    city_key = 0
    #  print('[Debug] city_key=', city_key)
    # 遍历市级城市列表信息
    for city_tr in city_tr_list:
        if city_tr:
            city_a_info = city_tr.select('a')
            #  print('[Debug] city_a_info=', city_a_info)
            city_href = city_a_info[0].attrs['href']
            #  print('[Debug] city_href=', city_href)
            city_code = city_a_info[0].text[:6]
            #  print('[Debug] city_code=', city_code)
            city_name = city_a_info[1].text
            #  print('[Debug] city_name=', city_name)
            #  city_info = {'code': city_code, 'name': city_name}
            city_info = {'text': city_name,'value': city_code, }
            #  print('[Debug] city_info=', city_info)
            #  city_list.setdefault(city_key, city_info)
            city_list.append(city_info)
            #  print('[Debug] city_list=', city_list)
            city_key += 1
            #  print('[Debug] city_key=', city_key)
            print('[INFO]抓取市级信息：', city_info)
            # 区级
            #  get_area(city_href, city_code)
            #  area_list = []
            #  area_list = get_area(city_href, city_code)
            #  print('[Debug] area_list=',area_list)
            #  area_lists.append(area_list)
    # 赋值给数据返回
    ret_data = city_list
    #  print('[Debug] ret_data=', ret_data)
    return ret_data


# 程序主入口
if __name__ == "__main__":
    url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/'
    # 创建json目录
    #  json_folder = 'json2019/'
    #  if not os.path.exists(json_folder):
    #      os.makedirs(json_folder)
    print('[INFO]开始…')
    get_province('index.html')
